#ifndef CALSIM_CPP_
#define CALSIM_CPP_
#include "header.h"
bool comparator( const mypair& l, const mypair& r)
{
        return l.second > r.second;
}

double inverseInLink(double sim, int inlinkNum)
{
    return sim * (1/(1+log(1+inlinkNum)));
}

double getCosineTFIDF(map<int,float>& p, map<int,float>& q)
{
    double dotRet = 0.0;
    double squareP = 0.0;
    double squareQ = 0.0;
    map<int,float>::iterator it = p.begin();
    map<int,float>::iterator end = p.end();
    map<int,float>::iterator qEnd = q.end();
    map<int,float>::iterator tmp;
    for (; it!=end; ++it) {     
        squareP += it->second * it->second;
        tmp = q.find(it->first);
        if (tmp != qEnd) {
            dotRet += tmp->second * it->second;
        }
    }
    it = q.begin();
    end = q.end();
    for (; it!=end; ++it) {     
        squareQ += it->second * it->second;
    }
    if (squareP < 0.0000000000001 || squareQ < 0.00000000001) return 0.0;
    double ret = dotRet/sqrt(squareP * squareQ);
    //std::cout << dotRet <<" "<< squareP <<" "<<squareQ << endl;
    if (ret != ret) return 0.0; //if ret is -nan ,then return 0.0
    else return ret;
}

double getCosine(vector<float>& p, vector<float>& q, int featureStart, int featureEnd)
{
    double dotRet = 0.0;
    double squareP = 0.0;
    double squareQ = 0.0;
    for (int i=featureStart; i<=featureEnd; ++i) {
        dotRet += p[i] * q[i];
        squareP += p[i] * p[i];
        squareQ += q[i] * q[i];
    }
    if (squareP < 0.000000001 || squareQ < 0.000000001) return 0.0;
    double ret = dotRet/sqrt(squareP * squareQ);
    if (ret != ret) return 0.0; //if ret is -nan ,then return 0.0
    else return ret;
}

//get the k-max largest similarity items in the array  using the minimum heap
vector<mypair> getKmax(vector<mypair>& array, int K)
{   
    int arraySize = array.size();
    vector<mypair> heapTmp;
    double tmp = 0.0;
    for(int i=0; i < arraySize; ++i) {
        if (i < K) {
            heapTmp.push_back(array[i]);
            if (i != K-1) continue;
        }
        if(i == K-1) {
            make_heap(heapTmp.begin(), heapTmp.end(), comparator);
        }   
        else if (i > K-1) {
            if (i == K) tmp = heapTmp[0].second;            
            if (tmp < array[i].second) {
                heapTmp[0].first = array[i].first;
                heapTmp[0].second = array[i].second;
                make_heap(heapTmp.begin(), heapTmp.end(), comparator);
                tmp = heapTmp[0].second;
            }
        }
    }   
    return heapTmp;
}

void storeSimAndNeigh(int qId, vector<mypair> &sim, int isSaveDb, 
        string simFun, ofstream &out, RelatedQuestions &relateQuestions,
        SimTable &simTable)
{
    vector<mypair> knns;
    if (1 == isSaveDb) {
        knns = getKmax(sim, SAVE_NEIGHB_NUM);
    }
    else {
        knns = getKmax(sim, NEIGHB_NUM);
    }
    //sort the sim array
    sort_heap(knns.begin(), knns.end(), comparator);
    int knnsSize = knns.size();
    ostringstream os;
    int neighbNum = 0;
    if (1 == isSaveDb) {
        neighbNum = SAVE_NEIGHB_NUM;
    } else {
        neighbNum = NEIGHB_NUM;
    } 
    for (int k = 0; k < knnsSize; ++k) {
        if (k < NEIGHB_NUM) {
            //store the k-nearest neighbours to file
            if (0==k) out << knns[k].first << "@" << knns[k].second;
            else out << "," << knns[k].first << "@" << knns[k].second;
        }
        if (k >=neighbNum) break;
        if (1 == isSaveDb) {
            if (0==k) os << knns[k].first << "@" << knns[k].second;
            else os << "," << knns[k].first << "@" << knns[k].second;
        }
    }
    if (1 == isSaveDb) {
        //store the k-nearest neighbours to db
        simTable.insertItem(qId, os.str().c_str());
    }
   //store the first result to file for the next step 
       //store the k-nearest neighbours to file
       //out << os.str(); 
    //if we get the nearest neighbour, we should update the inlink num 
    if ("IIL" == simFun) {
        relateQuestions.updateRelatedQuestions(qId, knns);
    }
    knns.clear();
}

void calSimAndSave(vector< vector<float> > &matrix, int itemNum, const char* dstFile, 
        int start, int end, string simFun, RelatedQuestions &relateQuestions)
{
    cout << "begin calculate the similarity of lsa:" << endl;
    ofstream out(dstFile);
    int docNum = itemNum;
    vector<mypair> sim(docNum);
    double tmpSim = 0.0; int i = 0, j = 0;
    int tmpInlinkNum = 0;
    int isSaveDb = atoi(g_config.Value("database","isSaveDb").c_str());

    //是否使用聚类结果计算相似度, 即是否只在lsa聚类的结果集中计算相似度
    int isUseCluster =  atoi(g_config.Value("section_1","isUseCluster").c_str());
    int isSample = atoi(g_config.Value("section_1","isSample").c_str()); //是否抽样计算计算相似度 
    const char* tableName = g_config.Value("database","table_name").c_str();
    SimTable simTable(&g_db, tableName);
    Cluster cluster;
    SampleQuestion sampleQuestion;
    if (1 == isUseCluster) {
        cluster.load(g_config.Value("cluster","clusterMap").c_str(),
        g_config.Value("cluster","clusterSet").c_str(), itemNum);
    }
    if (1 == isSample) {
        sampleQuestion.load(g_config.Value("section_1", "id_doc_map").c_str(), itemNum);
    }
    for (i=start; i<=end; ++i) {
        if (1 == isSample) { //是否抽样获取
            if(!sampleQuestion.isSampledQuestionId(i)) continue; //如果是抽样获取，且没有抽中，则跳到下一个
        }
        out << i << " ";
        //计算相似度的时候只与同类的item计算，如何保持兼容？有必要兼容，因为有可能只考虑同类
        //会降低相关度
        for (j=1; j<=docNum; ++j) {
        //vector<int> clusterSet = cluster.getClusterByqId(i);
        //for (size_t k=0; k < clusterSet.size(); ++k) {
        //    j = clusterSet[k];
            if (i == j) {
                continue;//tmpSim = 0.0; //make sure i will not similar with itself
            } else {
                tmpSim = getCosine(matrix[i], matrix[j], 0, FEATURE_NUM-1);
                if ("IIL" == simFun) {
                    tmpInlinkNum = relateQuestions.getInlink(j);
                    tmpSim = inverseInLink(tmpSim, tmpInlinkNum);
                }
                if (tmpSim != 0) {
                    sim.push_back(make_pair(j, tmpSim));
                }
            }
        }
        if (sim.size() > 0) { 
            storeSimAndNeigh(i, sim, isSaveDb, simFun, out, relateQuestions, simTable);
        }
        out << endl;
        sim.clear();
    }   
    out.close();
    cout << "end calculate the similarity of lsa!" << endl;
}

void calTFIDFSimAndSave(vector< map<int, float> >& matrix, int itemNum, const char* dstFile, 
        int start, int end, string simFun, RelatedQuestions &relateQuestions)
{
    cout << "begin calculate the similarity of tf-idf:" << endl;
    ofstream out(dstFile);
    int docNum = itemNum;
    double tmpSim = 0.0;
    int j = 0;
    vector<mypair> sim(docNum);  //存临时相似度
    int isSaveDb = atoi(g_config.Value("database","isSaveDb").c_str()); //是否存储到数据库
    //是否使用聚类结果计算相似度, 即是否只在lsa聚类的结果集中计算相似度
    int isUseCluster =  atoi(g_config.Value("section_1","isUseCluster").c_str());
    int isSample = atoi(g_config.Value("section_1","isSample").c_str()); //是否抽样计算计算相似度 
    const char* tableName = g_config.Value("database","table_name").c_str();
    string mongoHost = g_config.Value("database", "mongo_host"); //mongo host
    string mongoCollection = g_config.Value("database","mongo_collection"); //mongo collection

    SimTable simTable(&g_db, tableName);
    Cluster cluster;
    SampleQuestion sampleQuestion;
    if (1 == isUseCluster) {
        cluster.load(g_config.Value("cluster","clusterMap").c_str(),
        g_config.Value("cluster","clusterSet").c_str(), itemNum);
    }
    if (1 == isSample) {
        sampleQuestion.load(g_config.Value("section_1", "id_doc_map").c_str(), itemNum);
    }
    for (int i=start; i<=end; ++i) {
        if (1 == isSample) { //是否抽样获取
            if(!sampleQuestion.isSampledQuestionId(i)) continue; //如果是抽样获取，且没有抽中，则跳到下一个
        }
        out << i << " ";
        sim.clear();
        //for (j=1; j<=docNum; ++j) {
        vector<int> clusterSet;
        getBoolSearchRet(matrix[i], clusterSet, mongoCollection, mongoHost); 
        cout << i << '\t' << clusterSet.size() << endl;
        for (size_t k=0; k < clusterSet.size(); ++k) {
            j = clusterSet[k];
            if (i == j) {
                continue;//tmpSim = 0.0; //make sure i will not similar with itself
            } else {
		//cout <<i << '\t' << j << endl;
                tmpSim = getCosineTFIDF(matrix[i], matrix[j]);
                //cout<< i << "   " << j << "    " << tmpSim << endl;
                if ("IIL" == simFun) {
                    int inlinkNum = relateQuestions.getInlink(j);
                    tmpSim = inverseInLink(tmpSim, inlinkNum);
                }
                if (tmpSim != 0) {
                    sim.push_back(make_pair(j, tmpSim));
                }
            }
        }
        if (sim.size() > 0) {
            storeSimAndNeigh(i, sim, isSaveDb, simFun, out, relateQuestions, simTable);
        }
        out << endl;
    }   
    out.close();
    cout << "end calculate the similarity of tf-idf!" << endl;
}
#endif //CALSIM_CPP_
